#include <fstream>
#include <algorithm>
#include <stdexcept>
#include <vector>
#include <deque>
#include <string>
#include <iostream>


#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/foreach.hpp>



#define DEBUG_LEVEL 6
#include <debug_func.h>

#include <open_fstream.h>
#include <set_mark_diff.h>
#include <portable_timer.h>
#include <progress_indicator.h>
#include <stlhelper.h>
#include <merge_unique.h>
#include <ultoa.h>
#include <print_error.h>
#include <open_stream_name_with_substitutions.h>
#include <seq_pairs.h>
#include <crc.h>
#include <digest.h>
#include "get_arg.h"


using std::vector;
using std::deque;
using std::ofstream;
using std::ifstream;
using std::ostream;
using std::string;
using std::runtime_error;
using std::cerr;
using std::cin;
using std::cout;
using std::istream;
using std::sort;
using std::streampos;


//
// hash plus field which says whether the hash is from file 1 or 2
//
struct t_digest_used
	: public t_digest
{
	streampos	file_pos;
	t_digest_used(streampos file_pos_ = 0): 
			file_pos(file_pos_){}
	bool operator ==(const t_digest_used& o) const ;
	bool operator <(const t_digest_used& o) const ;
};
inline bool t_digest_used::operator ==(const t_digest_used& o) const
{
	return static_cast<const t_digest&>(*this) == static_cast<const t_digest&>(o);
}
inline bool t_digest_used::operator <(const t_digest_used& o) const
{
	return static_cast<const t_digest&>(*this) < static_cast<const t_digest&>(o);
}
struct t_sort_by_digest_file_pos
{
	bool operator()(const t_digest_used& a, const t_digest_used& b)
	{
		if (static_cast<const t_digest&>(a) == static_cast<const t_digest&>(b))
			return a.file_pos < b.file_pos;
		return static_cast<const t_digest&>(a) < static_cast<const t_digest&>(b);
	}
};

struct t_sort_by_file_pos
{
	bool operator()(const t_digest_used& a, const t_digest_used& b)
	{
		return a.file_pos < b.file_pos;
	}
};


//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888

//		Main

//8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888



//________________________________________________________________________________________

//		open_output_file

//________________________________________________________________________________________
bool open_output_file(ofstream& os, const string& file_name)
{
	if (!file_name.length())   
		return false;
	namespace fs = boost::filesystem;
	if (fs::exists(file_name) && fs::is_directory(file_name))
		throw std::runtime_error(file_name + " already exists and is a directory not a file.");
	os.open(file_name.c_str());
	if (!os)
		throw std::runtime_error("Could not open " + file_name);
	return true;
}





//________________________________________________________________________________________

//		get_ids_from_file

//________________________________________________________________________________________
void get_ids_from_file(	ifstream& ifs, const string& file_name, istream::pos_type file_size, 
						ostream& summary, deque<t_digest_used>& id_digests,
   						unsigned file_num, t_program_args& args)
{
	args.VERBOSE() << "\tParsing ids from [" << file_name << "]...\n";

	unsigned DOTS_SIZE = 100000;
	t_progress_indicator dots(args.VERBOSE(),  DOTS_SIZE);
	dots.use_timer();
	unsigned cnt_line = 0;

	//
	// handle any header
	//
	if (seq_pairs::skip_header(ifs))
		cnt_line += 2;

	try
	{
		streampos file_pos = ifs.tellg();
		string line;
		while (getline(ifs, line))
		{
			++cnt_line;
			++dots;
			if (line.empty())
			{
				file_pos += 1;
				continue;
			}
			
			// split out IDs unless comment line
			string ids;
			if (!seq_pairs::parse_normalized_ids_sub_string(line, ids, args.comment_letters))
			{
				file_pos += line.length() + 1;
				continue;
			}

			// calculate and save hash
			id_digests.push_back(t_digest_used(file_pos));
			crc_hash(ids, id_digests.back());

			if (dots.value() % (DOTS_SIZE * 10) == 0)
			{
				unsigned dots_needed = unsigned(dots.value() * 1.0 *
												file_size / file_pos);
				dots_needed += DOTS_SIZE;
				dots.set_limit(dots_needed);
			}
			file_pos += line.length() + 1;
		}
	}
	catch(std::runtime_error& e)
	{
		throw std::runtime_error(string(e.what()) + 
								 " on line #" + ultoa(cnt_line) + " of " + file_name);
	}
	//if (!cnt_line)
	//	throw std::runtime_error("No sequence pair data supplied in [" + file_name + "].");


	dots.finish();
	
	args.VERBOSE(2) << "\t\t" << id_digests.size() << " identifier pairs read from [" << file_name << "].\n";
	summary << std::setw(10) << id_digests.size() << "\tidentifier pairs from [" << file_name << "].\n";
	args.VERBOSE(2) << "\t\tSorting ID list...\n";
	sort(id_digests.begin(), id_digests.end(), t_sort_by_digest_file_pos());
	args.VERBOSE(2) << "\t\tRemoving duplicates...\n";
	remove_duplicates(id_digests);
	args.VERBOSE() << "\t\t" << id_digests.size() << " unique identifier pairs from [" << file_name << "].\n";
	summary << std::setw(10) << id_digests.size() << "\tunique identifier pairs from [" << file_name << "].\n";

}

//________________________________________________________________________________________

//		t_ostreams

//________________________________________________________________________________________
class t_ostreams
{
public:
	t_ostreams(t_program_args& args_)
		: 
		args(args_),
		cnt_shared(0),
		cnt_merged(0)
	{
		// save file names
		file_name[0]			= args.file_1;
		file_name[1]			= args.file_2;
		file_name_full[0]		= args.file_1_full;
		file_name_full[1]		= args.file_2_full;
		file_name_shared		= args.file_shared;      
		file_name_shared_full	= args.file_shared_full; 
		file_name_merged		= args.file_merged;      
		file_name_merged_full	= args.file_merged_full; 
		// open files
		for (unsigned i = 0; i <= 1; ++i)
		{
			do_file[i]			= open_output_file(file[i],				file_name[i]);
			do_file_full[i]		= open_output_file(file_full[i],		file_name_full[i]);
		}
		do_file_shared			= open_output_file(file_shared,			file_name_shared);
		do_file_shared_full 	= open_output_file(file_shared_full,	file_name_shared_full);
		do_file_merged			= open_output_file(file_merged,			file_name_merged);
		do_file_merged_full		= open_output_file(file_merged_full,	file_name_merged_full);
		cnt[0] = 
		cnt[1] = 0;
	}
	t_program_args& args;
	bool		any_shared_or_merged;
	bool		do_file[2];
	bool		do_file_full[2];
	bool		do_file_shared;
	bool		do_file_merged;
	bool		do_file_shared_full;
	bool		do_file_merged_full;
	ofstream	file[2];
	ofstream	file_full[2];
	ofstream	file_shared;
	ofstream	file_shared_full;
	ofstream	file_merged;
	ofstream	file_merged_full;
	string		file_name[2];
	string		file_name_full[2];
	string		file_name_shared;
	string		file_name_shared_full;
	string		file_name_merged;
	string		file_name_merged_full;
	unsigned  cnt[2];
	unsigned  cnt_shared;
	unsigned  cnt_merged;
};
//________________________________________________________________________________________

//		t_counts

//________________________________________________________________________________________
class t_counts
{
public:
	t_counts(unsigned cnt_unique1, unsigned cnt_unique2,
			 unsigned cnt_shared_)
		: 
		cnt_shared(cnt_shared_),
		cnt_merged(cnt_shared + cnt_unique1 + cnt_unique2)
	{
		cnt[0] = cnt_unique1;
		cnt[1] = cnt_unique2;
	}
	unsigned  cnt[2];
	unsigned  cnt_shared;
	unsigned  cnt_merged;
};


//________________________________________________________________________________________

//		print_matching

//________________________________________________________________________________________
inline void print_file_name_helper(std::ostream& os, const string& file_name)
{
	if (file_name.length())
		os << "\t\t [" << get_lastdir_filename(file_name) << "]\n";
}


void print_matching_unique(t_ostreams&			strms, 
							ifstream&			ifs,
							const string&		file_name, 
							deque<streampos>&	unique_filepos, 
							unsigned			file_num, 
							t_program_args&		args)
{
	if (!strms.do_file[file_num] && 
		!strms.do_file_full[file_num] && 
		!strms.do_file_merged_full &&
		!strms.do_file_merged)
		return;

	if (unique_filepos.empty())
	{
		args.VERBOSE()	<< "\tNo identifiers unique to ["
						<< get_lastdir_filename(file_name) << "]\n";
		return;
	}

	
	// sort by file position
	// sort by file position
	args.VERBOSE()	<< "\tSorting to read unique identifiers sequentially...\n";
	sort(unique_filepos.begin(),  unique_filepos.end());
	args.VERBOSE()	<< "\tPrinting identifiers unique in ["
					<< get_lastdir_filename(file_name) << "] to:\n";

	print_file_name_helper(args.VERBOSE(), strms.file_name[file_num]);
	print_file_name_helper(args.VERBOSE(), strms.file_name_full[file_num]);
	print_file_name_helper(args.VERBOSE(), strms.file_name_merged);
	print_file_name_helper(args.VERBOSE(), strms.file_name_merged_full);

	// go back to start
	ifs.clear();
	ifs.seekg(0, ifstream::beg);


	t_progress_indicator dots(args.VERBOSE(), unique_filepos.size() / 100 + 1);
	dots.set_limit(unique_filepos.size());
	dots.use_timer();

	//
	// Go to each file position and read
	//
	streampos curr_pos = 0;
	string line;
	try
	{
		BOOST_FOREACH(streampos sp, unique_filepos) 
		{
			if (curr_pos != sp)
			{
				ifs.seekg(sp, std::ios::beg);
				curr_pos = sp;
			}
			if (!getline(ifs, line))
				throw std::runtime_error("Could not read from position #" + 
										 ultoa(sp) + " of " + file_name);
			if (line.empty())
				throw std::runtime_error("Unexpected empty line at position #" + 
										 ultoa(sp) + " of " + file_name);
			curr_pos += line.length() + 1;
			++dots;
	
			// split out IDs unless comment line
			string ids;
			if (!seq_pairs::parse_normalized_ids_sub_string(line, ids, args.comment_letters))
				throw std::runtime_error("Unexpected comment line at position #" + 
										 ultoa(sp) + " of " + file_name);
			
			if (strms.do_file[file_num])
				strms.file[file_num] << ids << "\n";
			if (strms.do_file_full[file_num])
				strms.file_full[file_num] << line << "\n";
	
			// either
			if (strms.do_file_merged)
				strms.file_merged << ids << "\n";
			if (strms.do_file_merged_full)
				strms.file_merged_full << line << "\n";
	
		}
	}
	catch(std::runtime_error& e)
	{
		throw std::runtime_error(string(e.what()) + 
								 " at position #" + ultoa(curr_pos) + " of " + file_name +
								 "\n[" + line + "]");
	}
}


void print_matching_shared(	t_ostreams&			strms, 
							ifstream&			ifs,
							const string&		file_name, 
							deque<streampos>&	shared_filepos, 
							t_program_args&		args)
{
	if (!strms.do_file_shared && 
		!strms.do_file_shared_full && 
		!strms.do_file_merged_full &&
		!strms.do_file_merged)
		return;

	if (shared_filepos.empty())
	{
		args.VERBOSE()	<< "\tNo shared identifiers\n";
		return;
	}

	// sort by file position
	args.VERBOSE()	<< "\tSorting to read shared identifiers sequentially...\n";
	sort(shared_filepos.begin(),  shared_filepos.end());
	args.VERBOSE()	<< "\tPrinting shared identifiers from ["
					<< get_lastdir_filename(file_name) << "] to:\n";
	print_file_name_helper(args.VERBOSE(), strms.file_name_shared);
	print_file_name_helper(args.VERBOSE(), strms.file_name_shared_full);
	print_file_name_helper(args.VERBOSE(), strms.file_name_merged);
	print_file_name_helper(args.VERBOSE(), strms.file_name_merged_full);

	// go back to start
	ifs.clear();
	ifs.seekg(0, ifstream::beg);


	t_progress_indicator dots(args.VERBOSE(), shared_filepos.size() / 100 + 1);
	dots.set_limit(shared_filepos.size());
	dots.use_timer();

	//
	// Go to each file position and read
	//
	streampos curr_pos = 0;
	string line;
	try
	{
		BOOST_FOREACH(streampos sp, shared_filepos) 
		{
			if (curr_pos != sp)
			{
				ifs.seekg(sp, std::ios::beg);
				curr_pos = sp;
			}
			if (!getline(ifs, line))
				throw std::runtime_error("Could not read from position #" + 
										 ultoa(sp) + " of " + file_name);
			if (line.empty())
				throw std::runtime_error("Unexpected empty line at position #" + 
										 ultoa(sp) + " of " + file_name);
			curr_pos += line.length() + 1;
			++dots;

			ifs.seekg(sp, std::ios::beg);
			string line;
			if (!getline(ifs, line))
				throw std::runtime_error("Could not read from position #" + 
										 ultoa(sp) + " of " + file_name);
			if (line.empty())
				throw std::runtime_error("Unexpected empty line at position #" + 
										 ultoa(sp) + " of " + file_name);
			++dots;
	
			// split out IDs unless comment line
			string ids;
			if (!seq_pairs::parse_normalized_ids_sub_string(line, ids, args.comment_letters))
				throw std::runtime_error("Unexpected comment line at position #" + 
										 ultoa(sp) + " of " + file_name);
			
			if (strms.do_file_shared)
				strms.file_shared << ids << "\n";
			if (strms.do_file_shared_full)
				strms.file_shared_full << line << "\n";
	
			// either
			if (strms.do_file_merged)
				strms.file_merged << ids << "\n";
			if (strms.do_file_merged_full)
				strms.file_merged_full << line << "\n";
	
		}
	}
	catch(std::runtime_error& e)
	{
		throw std::runtime_error(string(e.what()) + 
								 " at position #" + ultoa(curr_pos) + " of " + file_name +
								 "\n[" + line + "]");
	}
}






//________________________________________________________________________________________

//		print_summary

//________________________________________________________________________________________
void print_summary(t_counts& cnts, const string& file_name1, const string& file_name2, 
				   ofstream& summary, t_program_args& args)
{
		//
	// print results common to all
	//
	if (cnts.cnt[0])
		args.VERBOSE() << "\t\t" << cnts.cnt[0]
									<< "\tmatching results only found in [" << file_name1 << "]\n";
	summary << std::setw(10)  << cnts.cnt[0]
									<< "\tmatching results only found in [" << file_name1 << "]\n";
	if (cnts.cnt[1])
		args.VERBOSE() << "\t\t" << cnts.cnt[1]
									<< "\tmatching results only found in [" << file_name2 << "]\n";
	summary << std::setw(10)  << cnts.cnt[1]                            
									<< "\tmatching results only found in [" << file_name2 << "]\n";

	if (cnts.cnt_shared)
		args.VERBOSE() << "\t\t" << cnts.cnt_shared
									<< "\tmatching results found in both files\n";
	summary << std::setw(10)  << cnts.cnt_shared
									<< "\tmatching results found in both files\n";

	if (cnts.cnt_merged)
		args.VERBOSE() << "\t\t" << cnts.cnt_merged
									<< "\tmatching results found in one file or the other\n";
	summary << std::setw(10)  << cnts.cnt_merged
									<< "\tmatching results found in one file or the other\n";

}

struct t_op_save_filepos
{
	deque<streampos>& file_pos;
	t_op_save_filepos(deque<streampos>& file_pos_):
				file_pos(file_pos_)
	{}
	void operator()(const t_digest_used& du){	file_pos.push_back(du.file_pos);}
};

template<typename T> struct t_op_ignore
{
	void operator ()(const T&){}
};

//________________________________________________________________________________________

//		main

//________________________________________________________________________________________
int main(int argc, char* argv[])
{
	t_program_args args;

	try
	{

	debug_func_cerr(5);

	// don't use C streams
	std::ios::sync_with_stdio(false);


	// get command line arguments
	if (!process_command_line_options(argc, argv, args))
		return 1;


	//
	//	prepare summary stream
	//
	ofstream summary;
	if (!args.file_summary.empty())
	{
		summary.open(args.file_summary.c_str());
		if (!summary.good())
			throw std::runtime_error("Could not open " + args.file_summary);
	}

	//
	//	Get IDs from input file1
	//
	deque<t_digest_used> id_digests1;
	ifstream input_fs1;
	string short_file_name1 = get_file_name_substitutions("[FILENAME][EXT]", args.file_input1);
	istream::pos_type file_size1 = open_fstream(input_fs1, args.file_input1);
	get_ids_from_file(input_fs1, short_file_name1, file_size1, summary, id_digests1, 1, args);


	//
	//	Get IDs from input file2
	//
	deque<t_digest_used> id_digests2;
	ifstream input_fs2;
	string short_file_name2 = get_file_name_substitutions("[FILENAME][EXT]", args.file_input2);
	istream::pos_type file_size2 = open_fstream(input_fs2, args.file_input2);
	get_ids_from_file(input_fs2, short_file_name2, file_size2, summary, id_digests2, 2, args);


	//
	// find unique ids in each file
	//
	//args.VERBOSE() << "\tSorting identifier pairs from each file...\n";
	//sort(id_digests1.begin(), id_digests1.end());
	//sort(id_digests2.begin(), id_digests2.end());

	args.VERBOSE() << "\tFinding identifier pairs unique to each file...\n";
	deque<streampos> file_pos_shared;
	deque<streampos> file_pos_unique1;
	deque<streampos> file_pos_unique2;
	set_mark_overlap_cont(	id_digests1,  
							id_digests2,  
							std::less<t_digest_used>(), 
							t_op_save_filepos(file_pos_unique1), 
							t_op_save_filepos(file_pos_unique2), 
							t_op_save_filepos(file_pos_shared), 
							t_op_ignore<t_digest_used>());



	//
	//	Release memory
	//
	args.VERBOSE() << "\tRelease memory...\n";
	deque<t_digest_used>().swap(id_digests1);
	deque<t_digest_used>().swap(id_digests2);

	t_counts counts(file_pos_unique1.size(), 
					file_pos_unique2.size(), 
					file_pos_shared.size());

	//
	//	print matching
	//
	t_ostreams strms (args);
	print_matching_unique(strms, input_fs1, short_file_name1, file_pos_unique1, 0, args);
	if (counts.cnt[1] != 0)
		print_matching_unique(strms, input_fs2, short_file_name2, file_pos_unique2, 1, args);

	print_matching_shared(strms, input_fs1, short_file_name1, file_pos_shared, args);

	//
	//	print summary
	//
	print_summary(counts, short_file_name1, short_file_name2, summary, args);


	args.VERBOSE() << "\tAll finished.\n";

	}
	catch (std::exception& e)
	{
		std_print_error(args.CERR(), e.what());
		return 1;
	}

	return 0;
}

